from pymongo import MongoClient
import json
import pymongo
import os
client = MongoClient(port=27017)
db=client['tiktok']
ids=[]
dids=[]
col=''
for obj in db[col].find_one():
    if len(obj['text'])==0:
        ids.append(obj['_id'])
for fname in os.listdir('D:\\Work\\Tool\\tiktok\\TikToks\\'):
    if col in fname.lower():
        with open('D:\\Work\\Tool\\tiktok\\TikToks\\'+fname, 'r', encoding='utf-8', newline='\n') as filename_input:
            ranking=0
            for line in filename_input:
                z = json.loads(line)
                text=''
                stext=[]
                id=''
                if 'id' in z.keys():
                    id = z['id']

                    if 'stickerTextList' in z.keys():
                        for item in z['stickerTextList']:
                            if type(item['stickerText']) is list:
                                for st in item['stickerText']:
                                    stext.append(st)
                    elif 'stickersOnItem' in z.keys():
                        for item in z['stickersOnItem']:
                            if 'stickerText' in z.keys():
                                if type(item['stickerText']) is list:
                                    for st in item['stickerText']:
                                        stext.append(st)
                                else:
                                    stext.append(item['stickerText'])
                    text=z['desc']
                elif 'itemInfos' in z.keys():
                    id = z['itemInfos']['id']

                    if 'stickerTextList' in z.keys():
                        for item in z['stickerTextList']:
                            if type(item['stickerText']) is list:
                                for st in item['stickerText']:
                                    stext.append(st)
                    elif 'stickersOnItem' in z.keys():
                        for item in z['stickersOnItem']:
                            if 'stickerText' in z.keys():
                                if type(item['stickerText']) is list:
                                    for st in item['stickerText']:
                                        stext.append(st)
                                else:
                                    stext.append(item['stickerText'])
                    text=z['itemInfos']['text']
                ranking += 1
                if ranking > 2000:
                    break
                if id not in ids:
                    continue
                if id in ids and id in dids:
                    continue
                temp=obj
                temp['_id']=id
                temp['text_feature']['text']=text
                temp['text_feature']['stickerText']=stext
                try:
                    db[hashtag].insert_one(temp)
                    print('insert successful',id)
                except pymongo.errors.DuplicateKeyError:
                    print(id, 'duplicate')
                i+=1